package forecasting.converter.parser;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Scanner;

import forecasting.converter.DiscreteValue;
import forecasting.converter.FeatureType;
import forecasting.converter.RealValue;
import forecasting.converter.UnknownValue;

/**
 * This abstract class scans through a file and generates a list of attribute types for each columnn. Once
 * the attribute list is created, we create an output file. Subclasses must implemented {@link #createOutputFile()}
 * to generate output files based on their use cases. Output files can be generated for specific formats, or
 * for data scrubbing.
 */
public abstract class AbstractDatasetParserTemplate {

	/** 
	 * There are 840 features.
	 * TODO: This should be 839. Let's check if we should remove the last entry.
	 */
	public static final int EXPECTED_FEATURE_VECTOR_SIZE = 840;
	
	protected final boolean saveCategories;
	
	public AbstractDatasetParserTemplate(boolean saveCategories) {
		this.saveCategories = saveCategories;
	}
	
	/**
	 * This method generates the column attributes, and then creates an output file.
	 * @param inputFileName
	 * @param outputFileName
	 */
	public void run(final String inputFileName, final String outputFileName) {
		// read in the entire file and look at it.	
		final ArrayList<FeatureType> labels = generateColumnAttributeTypes(inputFileName);
		createOutputFile(labels, inputFileName, outputFileName);
	}
	
	/**
	 * This method generates column attributes and should be used by all parsing algorithms.
	 * @param inputFile
	 */
	private ArrayList<FeatureType> generateColumnAttributeTypes(final String inputFileName) {
		ArrayList<FeatureType> labels = inferFeatureTypes(inputFileName);
		
		if (saveCategories) {
			return cacheDiscreteValueCategories(inputFileName, labels);
		} else {
			return labels;
		}
	}

	/**
	 * Do a pass across the dataset to infer feature types.
	 * @param inputFileName
	 * @return labels
	 */
	private ArrayList<FeatureType> inferFeatureTypes(final String inputFileName) {
		System.out.println("Inferring feature types...");
		final Scanner scan = openFileScanner(inputFileName);
		
		//scan for parameter names.
		String attributeNames[] = null;
		if(scan.hasNext()) {
			final String line = scan.nextLine();
			attributeNames = line.substring(0, line.length() - 1).split("\t", -1);
			resolveConflicts(attributeNames);
			
			for(String attribute: attributeNames)
				if(attribute.contains("clas")  || attribute.contains("class"))
					System.out.println("Attributes contained class");
		}
		
		ArrayList<FeatureType> labels = new ArrayList<FeatureType>();
		
		// first pass through dataset, we generate column types.
		int sampleNum = 0;
		while(scan.hasNext()) {
			++sampleNum;
			
			final String line = scan.nextLine();
			
			// checking for empty rows in the input dataset.
			if (line.length() == 0) {
				System.err.println("Unexpected empty line encountered on sample #" + sampleNum);
				continue;
			}
			
			final String attributeValues[] = line.substring(0, line.length() - 1).split("\t", -1);
			
			if (attributeValues.length != EXPECTED_FEATURE_VECTOR_SIZE) {
				System.err.println("For sample #" + sampleNum + ", we did not receive the expected number of columns. column size = " + attributeValues.length);
				//System.exit(-1);
			}
			
			// Iterate through the current sample's feature vector.
			for(int i = 0; i < attributeValues.length; i++) {
				final String attributeName = attributeNames[i];
				final String attributeValue = attributeValues[i];
				
				// Check if an initial feature type has been created yet. If not, let's create one.
				if(labels.size() <= i) {
					// first pass
					final FeatureType attribute = FeatureType.createValue(attributeName, attributeValue);
					labels.add(attribute); 					
					continue;
				}
				
				// Check if a feature which was inferred to be a real value is actually a discrete value.
				final FeatureType currentLabel = labels.get(i);
				if (currentLabel instanceof RealValue) {
					if (!RealValue.isReal(attributeValue)) {
						final FeatureType attribute = FeatureType.createValue(attributeName, attributeValue);
						if (!(attribute instanceof UnknownValue)) {
							labels.set(i, attribute);
						}
					} else {
						((RealValue) currentLabel).registerValue(RealValue.convertToDouble(attributeValue));
					}
				}
				
				// Reconfigure UnknownValues if we see actual data
				if (currentLabel instanceof UnknownValue) {
					// following passes where label has not yet been determined
					final FeatureType attribute = FeatureType.createValue(attributeName, attributeValue);

					// Only set the attribute if it is not null.
					if (!(attribute instanceof UnknownValue))
						labels.set(i, attribute);
				} 
				
			}
		}
		
		if (labels.size() != EXPECTED_FEATURE_VECTOR_SIZE) {
			System.err.println("Unexpected number of labels. Actual = " + labels.size());
			// System.exit(-1);
		}
		
		// pass through labels, and assign a null label for unknown label types
		scan.close();
		return labels;
	}
	
	private void resolveConflicts(String attrNames[]) {
		if(saveCategories){//make sure values are unique
			HashSet<String> set = new HashSet<String>();
			for(int i = 0; i < attrNames.length; i++) {
				if(set.contains(attrNames[i])) {
					if(!set.contains(attrNames[i].toLowerCase())) {	//need to change the value due to collision
						set.add(attrNames[i].toLowerCase());
						attrNames[i] = attrNames[i].toLowerCase();
					}	else {
						System.err.println("both versions exist");
						System.exit(-3);
					}
				}	else
					set.add(attrNames[i]);
			}
		}
	}
	
	/**
	 * Do a pass across the dataset to infer feature types.
	 * @param inputFileName
	 * @return labels
	 */
	private ArrayList<FeatureType> cacheDiscreteValueCategories(final String inputFileName, final ArrayList<FeatureType> labels) {
		System.out.println("Caching discrete value categories...");
		final Scanner scan = openFileScanner(inputFileName);
		
		// Skip header line since we've already derived our labels.
		if(scan.hasNext()) {
			scan.nextLine();
		}
		
		// Iterate across dataset, caching discrete values.
		int sampleNum = 0;
		while(scan.hasNext()) {
			++sampleNum;
			
			final String line = scan.nextLine();
			
			// checking for empty rows in the input dataset.
			if (line.length() == 0) {
				System.err.println("Unexpected empty line encountered on sample #" + sampleNum);
				continue;
			}
			
			final String attributeValues[] = line.substring(0, line.length() - 1).split("\t", -1);
			
			if (attributeValues.length != EXPECTED_FEATURE_VECTOR_SIZE) {
				System.err.println("For sample #" + sampleNum + ", we did not receive the expected number of columns. column size = " + attributeValues.length);
				//System.exit(-1);
			}
			
			for(int i = 0; i < attributeValues.length; i++) {
				// Cache discrete value categories
				if (labels.get(i) instanceof DiscreteValue) {
					((DiscreteValue)labels.get(i)).addValue(attributeValues[i]);
				}
			}
		}
		
		if (labels.size() != EXPECTED_FEATURE_VECTOR_SIZE) {
			System.err.println("Unexpected number of labels. Actual = " + labels.size());
			//System.exit(-1);
		}
		
		// pass through labels, and assign a null label for unknown label types
		scan.close();
		return labels;
	}	

	/**
	 * Utility method that's used for generating a {@link Scanner} for an existing file.
	 * @param filename
	 * @return Scanner
	 */
	public static Scanner openFileScanner(final String filename) {
		final File file = new File(filename);
		if(!file.exists()) {
			System.err.println(filename + " does not exist");
			System.exit(-2);
		}
		
		try {
			return new Scanner(file);
		} catch (FileNotFoundException e1) {
			System.err.println("Cannot create scanner for " + filename);
			System.exit(-3);
		}
		return null;
	}
	
	/**
	 * This method is implemented by subclasses to create an output file based on the attribute parsing.
	 * @param labels attribute labels for each column.
	 * @param inputFileName
	 * @param outputFileName
	 */
	protected abstract void createOutputFile(final ArrayList<FeatureType> labels, final String inputFileName, final String outputFileName);
}
